import requests,time
from pyquery import PyQuery as pq
import redis

url = 'https://www.goodreads.com/genres/list'

response = requests.get(url)
html = pq(response.text)
genres = html("a.mediumText")
r = redis.Redis(host='localhost',port='6379',decode_responses=True)
#print(genres)
for vo in genres.items():
    suburl = 'https://www.goodreads.com/list/tag/' + str(vo('a').text())
    print(suburl)
    response = requests.get(suburl)
    html = pq(response.text)
    subgenres = html("div.listImgs")
    for vi in subgenres.items():
        tagurl = 'https://www.goodreads.com' + str(vi('a').attr('href'))
        print(tagurl)
        r.lpush('goodreads:start_urls', tagurl)
        
    time.sleep(5)
